#include <config/Config.h>
#include <config/NovelConfig.h>
#include <dao/Connections.h>
#include <gather/Gather.h>
#include <config/RuleManager.h>
#include <log/NpLogger.h>
//#include <analyser/Compar.h>
#include <utils/XTimer.h>
#include <string>
#include "Spider.h"

Spider::Spider()
{
  m_chapterList = new chapterInfo::ChapterInfo[SIZE_CHAPTER_LIST];
  Config::getInstance()-> initializeConfigs();
  Connections::getInstance()->init();
  m_hasVips = 0;
  m_repeat = 0;
}

Spider::~Spider()
{
  if(m_chapterList != NULL)
  {
    delete[] m_chapterList;
    m_chapterList = NULL;
  }
  for(compar::typeRank::iterator it= m_rank.begin(); it!=m_rank.end(); ++it)
  {
    delete it->second;
  }
  for(typeContents::iterator it = m_contentsList.begin();it!=m_contentsList.end();++it)
  {
    delete it->second;
  }
}

void Spider::initialize()
{
  m_daoArticle.setConn(Connections::getInstance()->getConnection(Config::getInstance()->getConfig()->flag+"target"));
  m_daoChapter.setConn(Connections::getInstance()->getConnection(Config::getInstance()->getConfig()->flag+"target"));
  m_daoAttach.setConn(Connections::getInstance()->getConnection(Config::getInstance()->getConfig()->flag+"target"));
  NovelConfig::getInstance()->setMaxArticleID(m_daoArticle.getArticleMaxID());
  NovelConfig::getInstance()->setMaxChapterID(m_daoChapter.getChapterMaxID());
  m_firstRule = RuleManager::getInstance()->getRule(Config::getInstance()->getConfig()->ruleflag);
  m_htmlString.reserve(SIZE_HTML_STRING);
}

void Spider::work()
{
  initialize();
  while(Config::getInstance()->getOffon())
  {
    m_repeat = 0;
    m_firstAnalyseArticleList.setRule(*m_firstRule);
    m_firstAnalyseArticle.setRule(*m_firstRule);
    m_firstAnalyseContents.setRule(*m_firstRule);
    m_firstAnalyseChapter.setRule(*m_firstRule);
    getNovelList(m_firstAnalyseArticleList, m_firstAnalyseArticle, m_firstAnalyseContents,m_firstAnalyseChapter);
  }
}

void Spider::getNovelList(AnalyseArticleList& analyseArticleList,
                          AnalyseArticle& analyseArticle,
                          AnalyseContents& analyseContents,
                          AnalyseChapter& analyseChapter)
{
  static string listUrl;
  string tmpUrl;
  listUrl = analyseArticleList.getNovelListUrl();
  for(int ipage =1;
      ipage< Config::getInstance()->getConfig()->maxpage
      && m_repeat <= Config::getInstance()->getConfig()->repeatmax;
      ++ipage)
  {
    tmpUrl.assign(listUrl);
    XString::replaceAll(tmpUrl,"(*)", boost::lexical_cast<string>(ipage));
    m_htmlString.assign(Gather::getInstance()->collect(tmpUrl.c_str()));
    analyseArticleList.setContent(m_htmlString);
    analyseArticleList.analyseUrlList();
    for(int articleIndex = 0;
        articleIndex< analyseArticleList.getSize()
        && m_repeat <= Config::getInstance()->getConfig()->repeatmax;
        ++articleIndex)
    {
      cout<<"index:"<<articleIndex<<endl;
      try
      {
        getNovel(analyseArticleList[articleIndex], analyseArticle);
        getContents(analyseArticle,analyseContents);
#ifdef FLAG_STORAGE
        storageNovel(analyseArticle);
        analyseContents.setArticle(analyseArticle.getArticle());
        analyseChapter.setArticle(analyseArticle.getArticle());
        if(storageChapters(analyseContents,analyseChapter))
        {
          //需要处理VIP章节的TXT
          completeArtcle();
        }
#endif
        usleep(Config::getInstance()->getConfig()->sleepbook * 1000);
      }
      catch(npexception::NpException ex )
      {
        char msg[512] = {0};
        sprintf(msg,"[%d] link :%s",ex , analyseArticleList[articleIndex].c_str());
        NpLogger::getInstance()->errLog(msg);
      }
    }
  }
}

void Spider::completeArtcle()
{
  articleInfo::ArticleInfo article;
  article.articleid = m_lastinfo.articleid;
  article.lastupdate = boost::lexical_cast<string>(XTimer::getDate());
  article.lastchapterid=m_lastinfo.lastchapterid;
  article.lastchapter = m_lastinfo.lastchapter;
  article.lastvolume = m_lastinfo.lastvolume;
  article.lastvolumeid = m_lastinfo.lastvolumeid;
  article.chapters = m_lastinfo.chapters;
  m_daoArticle.updateArticle(article);
}

void Spider::getNovel(const string& articleLink, AnalyseArticle& analyseArticle)
{
  try
  {
    m_htmlString.clear();
    //m_htmlString.assign(Gather::getInstance()->collect("http://www.qidian.com/Book/2239968.aspx"));
    m_htmlString.assign(Gather::getInstance()->collect(articleLink.c_str()));
    analyseArticle.setContent(m_htmlString);
    analyseArticle.analyseNovelInfo();
#ifdef DEBUG
    printNovelInfo(analyseArticle.getArticle());
#endif
  }
  catch(npexception::NpException ex)
  {
    cout<<ex<<endl;
  }
}

void Spider::getContents(AnalyseArticle& analyseArticle, AnalyseContents& analyseContents)
{
  analyseContents.setArticle(analyseArticle.getArticle());
  m_htmlString.assign(Gather::getInstance()->collect(analyseArticle.getArticle().contentsLink.c_str()));
  analyseContents.setContent(m_htmlString);
  analyseContents.analyseVolumeList();
#ifdef DEBUG
  for(int ci = 0 ; ci< analyseContents.getSize(); ++ci)
  {
    printChapter(analyseContents[ci]);
  }
#endif
}

void Spider::storageNovel(AnalyseArticle& analyseArticle)
{
  bool articleExists = false;
  static articleInfo::ArticleInfo articleinfo;
  m_daoArticle.getArticle(analyseArticle.getArticle().articlename, articleinfo)==1?articleExists=true:articleExists=false;
  if(!articleExists)
  {
    analyseArticle.setArticleID(NovelConfig::getInstance()->getMaxArticleID());
    m_daoArticle.addArticle(analyseArticle.getArticle());  //novel id is memory maxid
    gatherCover(analyseArticle.getArticle().cover, analyseArticle.getArticle().articleid);
  }
  else
  {
    analyseArticle.setArticle(articleinfo);
  }
}

void Spider::gatherCover(const string& coverLink, const string& bookID)
{
  if(coverLink.empty())
    return;

  int len;
  char* cover =  Gather::getInstance()->collectAPic(coverLink.c_str(),len);
  char path[128]={0};
  sprintf(path,"%s/files/article/image/%d/%s", Config::getInstance()->getConfig()->filepath.c_str(),
          boost::lexical_cast<int>(bookID)/SIZE_DIR_SPLIT_UNIT,bookID.c_str());
  m_file.checkDir(path);
  strcat(path,"/");
  strcat(path, bookID.c_str());
  strcat(path,".jpg");
  m_file.writeBinary(cover,len,path );
}

bool Spider::storageChapters(AnalyseContents& analyseContents, AnalyseChapter& analyseChapter)
{
  int chapterLen = m_daoChapter.getChapterList(analyseContents.getArticle(), m_chapterList);
  if(chapterLen == analyseContents.getSize())
  {
    m_repeat ++;
    return false;
  }
  //将原有的卷查询出来后，遍历一次所有的章节，将卷ID赋值
  static string lastVolumeID;
  lastVolumeID = "";
  char msg[256] = {0};

  for(int ci = 0 ; ci< analyseContents.getSize(); ++ci)
  {
    analyseContents[ci].articleid = analyseContents.getArticle().articleid;

    if( ci < chapterLen)//old
    {
      if(m_chapterList[ci].ctype == "1")//chapter is image
        analyseContents[ci].isnew = '1';
      else
        analyseContents[ci].isnew='0';

      analyseContents[ci].volumeid = m_chapterList[ci].volumeid;//old volume id
      analyseContents[ci].chapterid = m_chapterList[ci].chapterid;//old chapter id
    }
    else if(ci >= chapterLen)//new
    {
      analyseContents.setChapterID(ci, NovelConfig::getInstance()-> getMaxChapterID());
      if(analyseContents[ci].chaptertype =='1')//volume,chapter
      {
        lastVolumeID = m_chapterList[ci].chapterid;
      }
      else if(analyseContents[ci].chaptertype =='0' && !lastVolumeID.empty())
      {
        analyseContents[ci].volumeid = lastVolumeID;
      }
      m_daoChapter.addChapter(analyseContents[ci]);
    }
    if(ci == analyseContents.getSize() -1)
    {
      analyseContents.getArticle().lastchapterid = m_chapterList[ci].chapterid;
      analyseContents.getArticle().lastchapter = m_chapterList[ci].chaptername;
    }
    if(analyseContents[ci].chaptertype != '1' && analyseContents[ci].isnew != '0')
    {
      if( analyseContents[ci].isvip =='0')
      {
#ifdef GET_CHAPTER_CONTENT
        getChapter(analyseChapter, analyseContents[ci]);
#endif
        m_lastinfo.articleid = analyseContents.getArticle().articleid;
        m_lastinfo.lastchapterid = analyseContents[ci].chapterid;
        m_lastinfo.lastchapter = analyseContents[ci].chaptername;
        m_lastinfo.lastvolumeid = analyseContents[ci].volumeid;
        m_lastinfo.chapters ++;
      }
      else
      {
        m_hasVips++;
      }
      //getVipChapter(analyseContents);
    }
#ifdef DEBUG
    sprintf(msg,"[IN]%s,%s,%s",analyseContents[ci].articlename.c_str(),
            analyseContents[ci].chapterid.c_str(),analyseContents[ci].chaptername.c_str());
    //printf("%s\r",msg);
    cout<<msg<<endl;
    NpLogger::getInstance()-> infoLog(msg);
#endif
  }
  return true;
}

void Spider::getChapter(AnalyseChapter& analyseChapter, chapterInfo::ChapterInfo& chapter)
{
  analyseChapter.setContent(Gather::getInstance()->collect(chapter.contentsLink.c_str()));
  analyseChapter.analyseChapterInfo(chapter);
  if(analyseChapter.getAttachSize() == 0)
  {
    gatherChapterContent(chapter, analyseChapter.getContentTxt());
    if(chapter.ctype == "1")//原来为图片章节，现在换成文字章节，需更新数据库
    {
      chapter.ctype ="0";
      m_daoChapter.updateChapterType(chapter);
      m_daoAttach.updateAttachInvalid(chapter.articleid);
    }
  }
  else if(analyseChapter.getAttachSize() >0)
  {
    chapter.ctype ="1";
    m_daoChapter.updateChapterType(chapter);
    //将标记类型更新为图片
    for(int ai = 0 ;ai<analyseChapter.getAttachSize(); ++ai)
    {
      getAttachImage(analyseChapter[ai]);
      m_daoAttach.addAttach(analyseChapter[ai]);
    }
  }
}

void Spider::gatherChapterContent(const chapterInfo::ChapterInfo& chapter, const string& content)
{
  char path[128]={0};
  sprintf(path,"%s/files/article/txt/%d/%s",
          Config::getInstance()->getConfig()->filepath.c_str(),
          boost::lexical_cast<int>(chapter.articleid)/SIZE_DIR_SPLIT_UNIT,
          chapter.articleid.c_str());
  m_file.checkDir(path);
  strcat(path,"/");
  strcat(path, chapter.chapterid.c_str());
  strcat(path,".txt");
  cout<<path<<endl;
  if(m_file.isExists(path))
  {
    m_file.deleteFile(path);
  }
  m_file.writeFile(content, path);
}

void Spider::getVipChapters(AnalyseContents& analyseContents)
{
  vector<string> secondRules;
  getSecondRules(secondRules);
  string novelUrl;
  for(size_t i=0;i<secondRules.size();++i)
  {
    addAnalyseContents(secondRules[i]);
    m_secondRule = RuleManager::getInstance()->getRule(secondRules[i]);
    //m_secondAnalyseArticleList.setRule(*m_secondRule);
    m_secondAnalyseArticle.setRule(*m_secondRule);
    AnalyseContents* tmpAnalyseContents = m_contentsList[secondRules[i]];
    tmpAnalyseContents->setRule(*m_secondRule);
    //m_secondAnalyseContents.setRule(*m_secondRule);
    m_secondAnalyseChapter.setRule(*m_secondRule);
    novelUrl = m_secondAnalyseArticle.analyseNovelSearchUrl(analyseContents.getArticle().articlename, novelUrl);
    m_htmlString = Gather::getInstance()->collect(novelUrl.c_str());
    m_secondAnalyseArticle.setContent(m_htmlString);
    novelUrl= m_secondAnalyseArticle.getNovelLink(novelUrl);
    getNovel(novelUrl, m_secondAnalyseArticle);

    compar::RankInfo *rank = new compar::RankInfo();
    rank->key = secondRules[i];
    //rank->remainTotal = Compar::getInstance()->getChapterDifferent(analyseContents,*tmpAnalyseContents);
    m_rank[rank->key] = rank;

    if(rank->remainTotal == 0)//完全匹配
    {
#ifdef FLAG_STORAGE
      storageNovel(m_secondAnalyseArticle);
      tmpAnalyseContents->setArticle(m_secondAnalyseArticle.getArticle());
      m_secondAnalyseChapter.setArticle(m_secondAnalyseArticle.getArticle());
      storageChapters(*tmpAnalyseContents,m_secondAnalyseChapter);
      return;
#endif
    }
  }

  //获取差距最小的站点
  string bestRule;
  //Compar::getInstance()->getBestRule(m_rank,bestRule);
}

void Spider::getSecondRules(vector<string> &rules)
{
  static string val;
  val = Config::getInstance()->getConfig()->secondrules;
  rules=XString::split(val,";");
}

void Spider::getAttachImage(attachInfo::AttachInfo& attach)
{

}

void Spider::printNovelInfo(const articleInfo::ArticleInfo& article)
{
  std::cout<<  article.articleid<<std::endl;
  std::cout<<  article.articlename <<std::endl;
  std::cout<<  article.author <<std::endl;
  std::cout<<  article.sortname<<std::endl;
  std::cout<<  article.cover<<std::endl;
  std::cout<<  article.contentsLink<<std::endl;
  std::cout<<  article.intro<<std::endl;
  NpLogger::getInstance()->infoLog( article.articleid.c_str());
  NpLogger::getInstance()->infoLog( article.articlename.c_str());
  NpLogger::getInstance()->infoLog(article.author.c_str());
  NpLogger::getInstance()->infoLog( article.sortname.c_str());
  NpLogger::getInstance()->infoLog(article.cover.c_str());
  NpLogger::getInstance()->infoLog( article.contentsLink.c_str());
  NpLogger::getInstance()->infoLog( article.intro.c_str());
}

void Spider::printChapter(const chapterInfo::ChapterInfo& chapter)
{
  // static  char str[1024] ={0};
  //  sprintf(str, "%s %s %s %s %s", chapter.articlename.c_str(),
  //          chapter.volumeid.c_str(),chapter.chapterid.c_str(),
  //          chapter.chapterorder.c_str(),chapter.chaptername.c_str());
  //  std:: cout<<str<<std::endl;
  //  NpLogger::getInstance()->infoLog( str);
}

void Spider::addAnalyseContents(string key)
{
  if(m_contentsList.find(key) == m_contentsList.end())
  {
    AnalyseContents* contents =new AnalyseContents();
    m_contentsList[key] = contents;
  }
}
